package com.yufei.infoExtractor.util;

import java.io.InputStream;
import java.io.InputStream;
import java.io.UnsupportedEncodingException;
import java.net.URLDecoder;
import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map;

import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.htmlparser.Node;
import org.htmlparser.Parser;
import org.htmlparser.nodes.TagNode;
import org.htmlparser.util.NodeIterator;
import org.htmlparser.util.NodeList;
import org.htmlparser.util.ParserException;

/**
 * created by @author jasstion at 2012-10-27
 */
public class HtmlUtil {
final static String[] picturePostfix={"png","jpg","gif","bmp","pcx","dcx","emf","JIF","JPE","JFIF","EPS"
	,"TIF","JPEG","RLE","DIB","PCD","DXF","ICO","WMF","TIFF","TGA"};
	private static Log mLog = LogFactory.getLog(HtmlUtil.class);

	public static String removeJSAndCss(String str) {
		if (str == null) {
			throw new IllegalArgumentException();
		}
		
		str = str.replaceAll("<STYLE[\\S\\s\\r\\n]{0,}?</STYLE>", "");
		str = str.replaceAll("<style[\\S\\s\\r\\n]{0,}?</style>", "");
	
		str = str.replaceAll("<SCRIPT[\\S\\s\\r\\n]{0,}?</SCRIPT>", "");
		str = str.replaceAll("<script[\\S\\s\\r\\n]{0,}?</script>", "");
		//str=str.replaceAll("<[\\S\\s\\r\\n]{0,}?>", "");
	
		return str;
	
	}
	public static String cleanHtml(String str){
		String result=null;
		result=HtmlUtil.removeJSAndCss(str);
		result=HtmlUtil.removeHtmlTag(result);
		return result;
		
	}
	public static String removeHtmlTag(String str) {
		if (str == null) {
			throw new IllegalArgumentException();
		}
		String[] specialHtmlChars={"&amp;","nbsp;","quot;","gt;"};
//remove htmlTag
		str=str.replaceAll("<[\\S\\s\\r\\n]{0,}?>", "");
	str=str.replaceAll("[\\s\\r\\n]", "");
	//对于一些特殊字符的过滤
	for(String tempStr:specialHtmlChars){
		str=str.replaceAll(tempStr, "");
	}
		return str;
	
	}
	public static String removeChineseChar(String source) {
		if (source == null) {
			throw new IllegalArgumentException();
		}
		
		return source.replaceAll("[\\u4E00-\\u9FA5]", "");
	}


	/** 
	 * @param html
	 * @return
	 * 选择与主题最为相关的网页内容
	 */
	public static CustomerHtmlNode removeHtmlNoise(String html){
		//String result=null;
		  Parser parser=new Parser(); 
		  CustomerHtmlNode maxSoreCustomerHtmlNode=null;
		  try {
			  //首先删除脚本和CSS
			  html=HtmlUtil.removeJSAndCss(html);
			  parser.setInputHTML(html);
	         NodeIterator elements=parser.elements();
		  CustomerHtmlNode rootCustomerMode=new CustomerHtmlNode();
		  Node tempNode=null;
		  Node currentMaxNode=null;
		  String tagName=null;
		 while(elements.hasMoreNodes()){
			 tempNode=elements.nextNode();
			 if(currentMaxNode!=null){
				 if(tempNode.toHtml().length()>currentMaxNode.toHtml().length()){
						currentMaxNode=tempNode;
					} 
			 }
			
			 if(currentMaxNode==null){
				 currentMaxNode=tempNode;
			 }
		 }
		if(currentMaxNode==null){
			mLog.debug("在对网页内容进行去噪处理的时候发生严重错误，请检查网页内容是否正确，此次处理直接忽略！");
		}
		HtmlUtil.createCustomerHtmlNode(currentMaxNode,rootCustomerMode, 0);
		maxSoreCustomerHtmlNode=rootCustomerMode.getCustomerHtmlNodeWithMaxScore();//HtmlUtil.getCustomerHtmlNodeByDepthAndOrderNum(rootCustomerMode,0,0);
      //  result=maxSoreCustomerHtmlNode.getHtmlContent();
      // result= cleanHtml(result);
			} catch (ParserException e) {
				// TODO Auto-generated catch block
				mLog.info(ExceptionUtil.getExceptionDetailsMessage(e));
			}
		return maxSoreCustomerHtmlNode;
		
	}
/**
 * @param node
 * @param rootHtmlNode
 * @param depth
 * 创建一个树，树中的每个节点都有两个属性：depth,orderNum(同一个节点的子节点的位置序列号，从零开始)
 */
public static  void createCustomerHtmlNode(Node node,CustomerHtmlNode rootHtmlNode,int depth){
	NodeList  nodes=null;
	CustomerHtmlNode nextCustomerHtmlNode=null;


	if(node==null){
		throw new IllegalArgumentException();
	}
	//根节点
	if(rootHtmlNode.getHtmlContent()==null){
		rootHtmlNode.setHtmlContent(node.toHtml());
	   rootHtmlNode.initHtmlSegmentStatistics();
		rootHtmlNode.setTopCustomerHtmlNode(null);
		depth=0;
		rootHtmlNode.setDepth(depth);
		rootHtmlNode.setOrderNum(0);
		nodes=node.getChildren();
		int preListSize=0;
		depth++;
		int orderNum=-1;
		//乱码的时候会出现
		if(nodes==null){
			return;
		}
		for(int i=0;i<nodes.size();i++){
		//	preListSize+=nodes.elementAt(i)
			orderNum++;
			nextCustomerHtmlNode=new CustomerHtmlNode(nodes.elementAt(i).toHtml());
			nextCustomerHtmlNode.setTopCustomerHtmlNode(rootHtmlNode);
			nextCustomerHtmlNode.setDepth(depth);
			nextCustomerHtmlNode.setOrderNum(orderNum);
			rootHtmlNode.getSubCustomerHtmlNode().add(nextCustomerHtmlNode);
			createCustomerHtmlNode(nodes.elementAt(i),nextCustomerHtmlNode,depth);
		}
		
	}
	//mLog.info(node.toHtml()+"\n");
	//mLog.info("***********************************************"+"\n");
	//mLog.info(rootHtmlNode.getHtmlSegmentStatistics().getPercentageOfContent()+"\n");
	
	if(node.getChildren()==null||node.getChildren().size()==0){
		
		return;

	}
	else{
		nodes=node.getChildren();
		int preListSize=0;
		depth++;
		int orderNum=-1;
		for(int i=0;i<nodes.size();i++){
		//	preListSize+=nodes.elementAt(i)
			orderNum++;
			nextCustomerHtmlNode=new CustomerHtmlNode(nodes.elementAt(i).toHtml());
			nextCustomerHtmlNode.setTopCustomerHtmlNode(rootHtmlNode);
			nextCustomerHtmlNode.setDepth(depth);
			nextCustomerHtmlNode.setOrderNum(orderNum);
			rootHtmlNode.getSubCustomerHtmlNode().add(nextCustomerHtmlNode);
			createCustomerHtmlNode(nodes.elementAt(i),nextCustomerHtmlNode,depth);
		}
		
	}
	rootHtmlNode.setContentSimilarityScore(rootHtmlNode.calculateContentSimilarityScore());
//	mLog.info(node.toHtml()+"\n");
	 //  mLog.info(rootHtmlNode.getContentSimilarityScore());

	//mLog.info("***********************************************"+"\n");

}

public static HtmlSegmentStatistics getHtmlSegmentStatistics(String htmlContent){
	HtmlSegmentStatistics htmlSegmentStatistics=null;
	int totalCount=htmlContent.length();
	int contentCharNum;
	int htmlCharNum;
	String plainText=HtmlUtil.removeJSAndCss(htmlContent);
	plainText=HtmlUtil.removeHtmlTag(plainText);
	contentCharNum=plainText.length();
	htmlCharNum=totalCount-contentCharNum;
	htmlSegmentStatistics=new HtmlSegmentStatistics(totalCount, htmlCharNum, contentCharNum);
	return htmlSegmentStatistics;
}

/**
 * @param rootCustomerHtmlNode
 * @param depth：树结构中的深度，root节点的深度为0
 * @param orderNum：某一深度上，某个特定index位置的元素（index也是从零开始）
 * @return
 * 获取某个深度某个位置的元素
 */
public static CustomerHtmlNode getCustomerHtmlNodeByDepthAndOrderNum(CustomerHtmlNode rootCustomerHtmlNode,int depth,int orderNum){
	CustomerHtmlNode result=null;
List<CustomerHtmlNode> list=new ArrayList<CustomerHtmlNode>();

	HtmlUtil.convertRootCustomerHtmlNodeToList(rootCustomerHtmlNode, list);
	Collections.sort(list);
	result=list.get(list.size()-1);
	return result;
}
public static void convertRootCustomerHtmlNodeToList(CustomerHtmlNode customerHtmlNode,List<CustomerHtmlNode> list){
	list.add(customerHtmlNode);
	int size = customerHtmlNode.getSubCustomerHtmlNode().size();
	if(size!=0){
		for(int i=0;i<size;i++){
			convertRootCustomerHtmlNodeToList(customerHtmlNode.getSubCustomerHtmlNode().get(i),list);
		}
	}
	else{
		return;
	}
}
/**
 * @param htmlContent
 * @return
 * 注意：返回的结果字符串可能含有单双引号或者不是完整的链接（需要补域名）
 */
public static  List<String> getImageUrls(String htmlContent){
	List<String> imagesUrls=new ArrayList<String>();
	//<img src=http://g1.ykimg.com/0100401F4650B2406016CD06E0B90FE94359C6-4FD2-473C-1FA7-C1B5470900C7replace="false">   
	String imageUrlPattern="<img[\\s\\S\\r\\n]{0,}src=(.*?)\\s";
	imagesUrls=PatternUtils.getListStrByRegex(htmlContent, imageUrlPattern);
	String tempPostfix=null;
	for(String imageUrl:imagesUrls){
		
		tempPostfix=getPostfixOfPictureUrl(imageUrl);
		if(CommonUtil.isEmptyOrNull(tempPostfix)){
			continue;
		}
		imageUrl=imageUrl.substring(0,imageUrl.lastIndexOf(tempPostfix));
		
		
	}
	return imagesUrls;
	
} 
public static  String encoderUrl(String url){
	String result = null;
	try {
		result = URLEncoder.encode(url, "UTF-8");
	} catch (UnsupportedEncodingException e) {
		// TODO Auto-generated catch block
		ExceptionUtil.getExceptionDetailsMessage(e);
	}
	return result;

}
public static  String decodeUrl(String url){
	String result = null;
	try {
		result = URLDecoder.decode(url, "UTF-8");
	} catch (UnsupportedEncodingException e) {
		// TODO Auto-generated catch block
		ExceptionUtil.getExceptionDetailsMessage(e);
	}
	return result;

}
public static String detectEncoding(InputStream inputStream){
	String encoding=null;
	//int readBytes=
	return encoding;
	
}
private static String getPostfixOfPictureUrl(String pictureUrl){
	String pictureUrlPostfix=null;
	for(String postfix:picturePostfix){
		if(pictureUrl.toLowerCase().contains(postfix.toLowerCase())){
			pictureUrlPostfix=postfix;
			break;
		}
		
	}
	return pictureUrlPostfix;
}
}
